import thread
import re
import os
import Queue
import uuid
import time
import pycurl
import sys
from urlparse import urlparse,urljoin,urlsplit,SplitResult
import logging
import threading

logger = logging.getLogger()


class CurlResponse(object):
    # Curl error codes
    CURLE_OPERATION_TIMEDOUT = 28

    def __init__(self, http_code=0, server_ip="", curl_err_code=None, curl_err_msg="", err_msg_http=""):
        self._http_code = http_code
        self._server_ip = server_ip
        self._curl_err_code = curl_err_code
        self._curl_err_msg = curl_err_msg
        self._err_msg_http = err_msg_http

        self._effective_url = None
        self._url = None
        self._uuid = None

    @property
    def url(self):
        return self._url

    @url.setter
    def url(self, v):
        self._url = v

    @property
    def uuid(self):
        return self._uuid

    @uuid.setter
    def uuid(self, v):
        self._uuid = v

    @property
    def http_code(self):
        return self._http_code

    @http_code.setter
    def http_code(self, v):
        self._http_code = v

    @property
    def server_ip(self):
        return self._server_ip

    @server_ip.setter
    def server_ip(self, v):
        self._server_ip = v

    @property
    def effective_url(self):
        return self._effective_url

    @effective_url.setter
    def effective_url(self, v):
        self._effective_url = v

    @property
    def curl_err_code(self):
        return self._curl_err_code

    @curl_err_code.setter
    def curl_err_code(self, v):
        self._curl_err_code = v

    @property
    def curl_err_msg(self):
        return self._curl_err_msg

    @curl_err_msg.setter
    def curl_err_msg(self, v):
        self._curl_err_msg = v

    @property
    def err_msg_http(self):
        return self._err_msg_http

    @err_msg_http.setter
    def err_msg_http(self, v):
        self._err_msg_http = v


class CurlWrapper(object):
    RE_PATTERN_IS_NTLM_PROXY = re.compile(r'Proxy-Authenticate:\s+NTLM', re.IGNORECASE)
    RE_PATTERN_IS_HTTP_407 = re.compile(r'Received HTTP code 407 from proxy after CONNECT', re.IGNORECASE)
    IPRESOLVE_WHATEVER = 0
    IPRESOLVE_V4_ONLY = 1
    IPRESOLVE_V6_ONLY = 2
    IPRESOLVE_V4_FIRST = 3
    IPRESOLVE_V6_FIRST = 4
    STATE_IPV4 = 0
    STATE_IPV6 = 1
    STATE_WHATEVER = 2
    RESOLVE_MAPPING = {STATE_IPV4: 'IPv4',
                       STATE_IPV6: 'IPv6',
                       STATE_WHATEVER: 'ANY'}
    PROXY_TYPE_MAPPING = {'http': pycurl.PROXYTYPE_HTTP,
                          'socks4': pycurl.PROXYTYPE_SOCKS4,
                          'socks5': pycurl.PROXYTYPE_SOCKS5}

    def __init__(self, total_timeout, connect_timeout, use_default_ssl_version=False):
        self._curl = pycurl.Curl()

        # Configuration
        self._total_timeout = total_timeout
        self._connect_timeout = connect_timeout
        self._max_retry_times = 2
        self._ip_resolve_type = self.IPRESOLVE_WHATEVER
        self._proxy_host = None
        self._proxy_port = None
        self._proxy_user = None
        self._proxy_passwd = None
        self._proxy_type = None
        self._resolve_retry_threshold = 3
        self._resolve_retry_timer = 60
        self._progress_update_threshold = 5
        self._force_abort = False

        # Enabled
        self._proxy_enabled = False
        self._progress_enabled = False
        self._resume_enabled = False
        self.enable_debug = False
        self.curl_verb_msg = []

        # Run-time status
        self._url = None
        self._header_list = None
        self._current_resolving_type = CurlWrapper.STATE_WHATEVER
        self._count_resolve_change_then_success = 0
        self._resolve_change_time = None
        self._progress_update_time = None
        self._resume_file_path = None
        self._retry_by_http_code_args = ()
        self._options = {}
        self._change_proxy = False
        self._initialize_args = ()
        self._add_to_job_queue_time = None
        self.priority = None

        # NTLM workaround
        self._is_ntlm = False
        self._pycurl_23_caused_by_ntlm = False
        self._proxy_407_error = False

        self._last_curl_response = None

        self._use_default_ssl_version = use_default_ssl_version

        # init
        self._get_curl_version()
        self._set_options()

        # callback
        self._download_id = None
        self._curl_response = CurlResponse()
        self._callback_done = None

        # handler signal
        self._reties = 0

        # Redirector
        self._uuid = uuid.uuid4()

    def __cmp__(self, other):
        # When curlwrapper objects are in PriorityQueue,
        #  order them by priority, then by object address.
        return cmp((self.priority, id(self)), (other.priority, id(other)))

    @property
    def uuid(self):
        return self._uuid

    @property
    def url(self):
        return self._url

    @url.setter
    def url(self, value):
        self._url = value

    @property
    def header_list(self):
        return self._header_list

    @header_list.setter
    def header_list(self, value):
        self._header_list = value

    @property
    def download_id(self):
        return self._download_id

    @download_id.setter
    def download_id(self, value):
        self._download_id = value

    @property
    def force_abort(self):
        return self._force_abort

    @force_abort.setter
    def force_abort(self, value):
        self._force_abort = value

    @property
    def callback_done(self):
        return self._callback_done

    @callback_done.setter
    def callback_done(self, value):
        self._callback_done = value

    @property
    def curl_response(self):
        return self._curl_response

    @curl_response.setter
    def curl_response(self, value):
        self._curl_response = value

    @property
    def http_code(self):
        return int(self._curl.getinfo(pycurl.HTTP_CODE))

    @property
    def primary_ip(self):
        return self._curl.getinfo(pycurl.PRIMARY_IP)

    @property
    def curl_err_code(self):
        return self._curl_response.curl_err_code

    @property
    def curl_err_msg(self):
        return self._curl_response.curl_err_msg

    @property
    def err_msg_http(self):
        return self._curl_response.err_msg_http

    @property
    def is_done(self):
        return self._reties > self._max_retry_times

    @property
    def is_succeed(self):
        return self.curl_response.http_code > 0 and \
            not self.curl_err_code and \
            not self.curl_err_msg

    @property
    def need_retry(self):
        """
        need retry when
        1. not 2xx
        2. not 4xx
        3. not (501, 505, 508, 511, 521)
        """
        return self._should_retry_by_http_code(self.http_code, *self._retry_by_http_code_args)

    def _get_curl_version(self):
        ver_dict = dict(item.split("/", 1) for item in pycurl.version.split(" ") if 'libcurl' in item)
        if 'libcurl' not in ver_dict:
            self._curl_version_major = 0
            self._curl_version_minor = 0
            self._curl_version_fix = 0
            return
        curl_ver_dict = ver_dict['libcurl'].split('.')
        self._curl_version_major = int(curl_ver_dict[0])
        self._curl_version_minor = int(curl_ver_dict[1])
        self._curl_version_fix = int(curl_ver_dict[2]) if len(curl_ver_dict) > 2 else 0

    def get_curl_object(self):
        return self._curl

    def set_option(self, key, value):
        self._options[key] = value
        self._curl.setopt(key, value)

    def unset_option(self, key):
        del self._options[key]
        self._curl.unsetopt(key)

    def pre_preform(self, url, header_list):
        if isinstance(url, unicode):
            url = url.encode('utf-8')

        # patch url if the path is empty
        url = self.patch_url(url)

        self._url = url
        self._header_list = header_list
        self._curl.setopt(pycurl.URL, self._url)

        self._check_resolving_for_adaptive_mode()

        self._check_resolving_for_proxy()

    def perform(self, url, header_list):
        self.pre_preform(url, header_list)

        ret_result = self._perform()
        res = (ret_result, self.curl_response)

        if res[0]:  # success
            logger.debug("Query url:%s successfully with %s resolving",
                         url,
                         CurlWrapper.RESOLVE_MAPPING[self._current_resolving_type])

            # if not success, counter should not be changed
            self._update_resolving_statistics()
        self._update_resolving_when_finish()

        return res

    def setup_curl(self):
        if self._change_proxy:
            self._renew_curl_object()
        self._workaround_ntlm_proxy()
        self._set_header()
        if self._resume_enabled:
            self._resume_from_file(self._resume_file_path)

    def pycurl_error_handling(self, curl_err_code, curl_err_msg):
        """
        pycurl error handling should be reusable for pycurl.perform and curlmulti
        @curl_err_code: the error code of curl perform
        @curl_err_msg: the errro msg of curl perform
        """
        retry = False

        error_response = CurlResponse()
        error_response.curl_err_code = curl_err_code
        error_response.curl_err_msg = curl_err_msg
        error_response.effective_url = self._curl.getinfo(pycurl.EFFECTIVE_URL)
        error_response.server_ip = self._curl.getinfo(pycurl.PRIMARY_IP)
        if self.http_code != 0:
            error_response.err_msg_http = " {0} code:{1}".format(CurlWrapper._get_protocol_from_url(self._url), self.http_code)

        # NTLM workaround:
        # Don't need to retry when authentication fail
        if self._proxy_407_error:
            logger.error("Received HTTP code 407 from proxy after CONNECT. url:%s", self._url)
            self._proxy_407_error = False
            error_response.http_code = 407
            error_response.curl_err_code = 56
            error_response.curl_err_msg = "Received HTTP code 407 from proxy after CONNECT."
            self.curl_response = error_response

        elif self._pycurl_23_caused_by_ntlm:
            # NTLM workaround:
            # If the proxy server is detected as a NTLM proxy, it should
            # renew the curl object and retry again.
            # Note that the retry count should not increase in this case

            # In ntlm workaround, should retry
            self._pycurl_23_caused_by_ntlm = False
            self._is_ntlm = True
            retry = True

        elif self.is_client_error(error_response.curl_err_code):
            # If http code is not 0 or None, it means the http request
            # is successful but client callback raised a pycurl.error.
            if self.need_retry:
                logger.debug("Receive http code %s, but curl error code: %s, retry (%s/%s).", self.http_code, error_response.curl_err_code, retry, self._max_retry_times)
                retry = True
            else:
                logger.debug("pycurl error due to client-side error. error code: %s, error msg: %s%s, url: %s", error_response.curl_err_code, error_response.curl_err_msg, error_response.err_msg_http, self._url)
                self.curl_response = error_response

        elif not self.should_retry_by_error_code(error_response.curl_err_code, error_response.curl_err_msg):
            if self._should_change_resolving(error_response.curl_err_code):
                # reset retry times if ip resolving is changed
                self._update_resolving_when_fail()
                logger.debug("Changed IP resolving to %s due to curl error. error code: %s, error msg: %s", CurlWrapper.RESOLVE_MAPPING[self._current_resolving_type], error_response.curl_err_code, error_response.curl_err_msg)

                self._last_curl_response = error_response
                retry = True
            else:
                if self._last_curl_response is None:
                    logger.error("pycurl error. error code: %s, error msg: %s%s, url: %s, effective_url: %s",
                                 error_response.curl_err_code,
                                 error_response.curl_err_msg,
                                 error_response.err_msg_http,
                                 self._url,
                                 error_response.effective_url)
                else:
                    logger.error("pycurl error. error code: %s, error msg: %s%s, url: %s, effective_url: %s, last_response: %s",
                                 error_response.curl_err_code,
                                 error_response.curl_err_msg,
                                 error_response.err_msg_http,
                                 self._url,
                                 error_response.effective_url,
                                 vars(self._last_curl_response))
                self.curl_response = error_response

        elif not self._check_resolving_for_hostname():
            error_response.curl_err_msg = "No need to perform curl query since the hostname of url(%s) doesn't match the resolving type(%s)" % (self._url, CurlWrapper.RESOLVE_MAPPING[self._current_resolving_type])

            logger.debug("%s" % error_response.curl_err_msg)
            self.curl_response = error_response
        else:
            self.curl_response = error_response
            retry = True

        if retry:
            logger.debug("pycurl error, retry (%s/%s). error code: %s, error msg: %s%s", self._reties, self._max_retry_times, error_response.curl_err_code, error_response.curl_err_msg, error_response.err_msg_http)
            # ntlm workaround should not add retry times
            if not self._pycurl_23_caused_by_ntlm:
                self._reties += 1

        return retry

    def _perform(self):
        self.curl_response = CurlResponse()

        def _exist_perform(ret):
            # ret: perform return result
            # is_done: exceed max retries
            # is_succeed: http code is good
            return not ret or self.is_done or self.is_succeed

        # @retry(quiet=True)
        def _do_perform():
            self._initialize(self.http_code, self.curl_err_code, self.curl_err_msg, *self._initialize_args)
            self.setup_curl()
            retry = False
            try:
                logger.debug('try perform')
                self._curl.perform()
                self.curl_response = CurlResponse(http_code=self.http_code, server_ip=self.primary_ip)
                if self.need_retry:
                    logger.debug("Receive http code %s, retry (%s/%s), it should retry.", self.http_code, self._reties, self._max_retry_times)
                    self._reties += 1
                    retry = True
                else:
                    logger.debug("pycurl done, http_code:%s, server_ip:%s", self.http_code, self.primary_ip)
            except pycurl.error as e:
                retry = self.pycurl_error_handling(e.args[0], e.args[1])

            return _exist_perform(retry)

        # curl perform
        logger.debug("Start to perform curl %s with %s" % (self.url, CurlWrapper.RESOLVE_MAPPING[self._current_resolving_type]))
        _do_perform()

        if self.is_succeed:
            logger.debug("Successfully to perform curl with %s" % CurlWrapper.RESOLVE_MAPPING[self._current_resolving_type])
        elif self._should_change_resolving(self.curl_err_code):
            # evaluate to change resolver if not success
            self._reties = 0
            self._update_resolving_when_fail()
            logger.warning("pycurl max retry times has reached, change to resolve type %s and retry" % CurlWrapper.RESOLVE_MAPPING[self._current_resolving_type])
            _do_perform()

        # If _should_retry_by_http_code is overwritten and all retry failed,
        # the return value should still be True because the curl query is successful
        if not self.is_succeed:
            if self.force_abort:
                logger.debug("pycurl has been aborted. http code:%s ,curl_err_code:%s, curl_err_msg:%s%s, url:%s", self.http_code, self.curl_err_code, self.curl_err_msg, self.err_msg_http, self._url)
            else:
                logger.error("pycurl max retry times has reached or resovling type didn't match. http code:%s ,curl_err_code:%s, curl_err_msg:%s%s, url:%s", self.http_code, self.curl_err_code, self.curl_err_msg, self.err_msg_http, self._url)
            return False

        logger.debug("Receive http code %s, success for url(%s) get repsonse: %s", self.http_code, self.url, vars(self.curl_response))
        return True

    @staticmethod
    def patch_url(url):
        return url
        # # patch ://
        # urlpatched = False
        # if '://' not in url:
        #     url = "//" + url
        #     urlpatched = True

        # # patch url path empty
        # parsed = urlparse(url)
        # if parsed.path:
        #     if urlpatched:
        #         return url[2:]
        #     else:
        #         return url
        # url_parts = list(parsed)
        # url_parts[2] = '/'  # index 2 -> path

        # if urlpatched:
        #     return (url_parts)[2:]
        # else:
        #     return urlunparse(url_parts)

    def debug_func(self, debug_type, debug_msg):
        logger.debug('{} {}'.format(debug_type, debug_msg))
        # if debug_type == pycurl.INFOTYPE_TEXT:
        #     if self.RE_PATTERN_IS_HTTP_407.search(debug_msg):
        #         self._proxy_407_error = True

        # if self.enable_debug:
        #     self.curl_verb_msg.append((debug_type, debug_msg))

    def is_client_error(self, err_code):
        # CURLE_WRITE_ERROR (23)
        # CURLE_READ_ERROR (26)
        # CURLE_OUT_OF_MEMORY (27)
        # CURLE_FUNCTION_NOT_FOUND (41)
        # CURLE_ABORTED_BY_CALLBACK (42)
        # CURLE_BAD_FUNCTION_ARGUMENT (43)
        # CURLE_INTERFACE_FAILED (45)
        # CURLE_UNKNOWN_OPTION (48)
        # CURLE_FILESIZE_EXCEEDED (63)
        return err_code in (23, 26, 27, 41, 42, 43, 45, 48, 63)

    def set_http_code_retry_callback(self, callback, *args):
        self._should_retry_by_http_code = callback
        self._retry_by_http_code_args = args

    def _should_retry_by_http_code(self, http_code, *args):
        return False

    def set_init_callback(self, callback, *args):
        self._initialize = callback
        self._initialize_args = args

    def _initialize(self, http_code, curl_err_code, curl_err_msg, *args):
        pass

    def set_proxy(self, enabled, proxy_host=None, proxy_port=None, proxy_type='http', proxy_user=None, proxy_passwd=None):
        # unset old status
        self._is_ntlm = False
        self._pycurl_23_caused_by_ntlm = False
        self._proxy_407_error = False

        # return if no changed
        if self._proxy_host == proxy_host and self._proxy_port == proxy_port and \
           self._proxy_type == proxy_type and self._proxy_user == proxy_user and \
           self._proxy_passwd == proxy_passwd and self._proxy_enabled == enabled:
            return

        # check if proxy type unsupported
        if proxy_type.lower() not in CurlWrapper.PROXY_TYPE_MAPPING:
            logger.error("Invalid proxy type(%s) detected. Proxy will not be set.", proxy_type)
            return

        # save new setting and mark proxy changeing, the new setting will be applied when next curl perform
        self._proxy_enabled = enabled
        self._proxy_host = proxy_host
        self._proxy_port = proxy_port
        self._proxy_user = proxy_user
        self._proxy_passwd = proxy_passwd
        self._proxy_type = proxy_type
        self._change_proxy = True

    def _set_proxy_setting(self):
        if not self._change_proxy:
            return
        self._change_proxy = False

        if not self._proxy_enabled:
            logger.debug("Proxy is disabled on curlwrapper [%s]" % (self))
            self._curl.setopt(pycurl.PROXY, '')
            return

        # apply new proxy setting
        logger.debug('Proxy is enabled with proxy_type(%s), proxy(%s:%d), username(%s) on curlwrapper [%s], fips_mode(%d)',
                     self._proxy_type,
                     self._proxy_host, self._proxy_port,
                     self._proxy_user,
                     self._curl,
                     'utility.is_fips_mode_enabled()')
        ptype = CurlWrapper.PROXY_TYPE_MAPPING[self._proxy_type.lower()]
        # self._curl.setopt(pycurl.PROXY, "[%s]" % self._proxy_host if utility.is_valid_ipv6(self._proxy_host) else self._proxy_host)
        self._curl.setopt(pycurl.PROXYPORT, self._proxy_port)
        self._curl.setopt(pycurl.PROXYTYPE, ptype)

        if self._proxy_user:
            self._curl.setopt(pycurl.PROXYUSERPWD, "%s:%s" % (self._proxy_user, self._proxy_passwd) if self._proxy_user else None)
        else:
            self._curl.unsetopt(pycurl.PROXYUSERPWD)

        # Set proxy authentication method
        self._set_proxy_auth()

    def _set_proxy_auth(self):
        proxy_auth = pycurl.HTTPAUTH_ANY

        # # When FIPS mode is enabled, http proxy only allow basic authentication
        # if utility.is_fips_mode_enabled():
        #     proxy_auth = pycurl.HTTPAUTH_BASIC | pycurl.HTTPAUTH_GSSNEGOTIATE

        # # FIPS mode is disabled, NTLM can be used
        # elif self._is_ntlm:
        #     proxy_auth = pycurl.HTTPAUTH_NTLM

        self._curl.setopt(pycurl.PROXYAUTH, proxy_auth)

    def set_retry(self, retry_times):
        if retry_times <= 0 or retry_times is None:
            self._max_retry_times = 0
        else:
            self._max_retry_times = retry_times

    def set_resume(self, enabled, file_path):
        self._resume_enabled = enabled
        self._resume_file_path = file_path

    def _resume_from_file(self, file_path):
        if os.path.exists(file_path):
            partial_file_size = os.path.getsize(file_path)
            if partial_file_size != 0:
                logger.debug("Detect partial file in cache, resume download from size %s", partial_file_size)
                self._curl.setopt(pycurl.RESUME_FROM, partial_file_size)
        else:
            self._curl.setopt(pycurl.RESUME_FROM, 0)

    def set_progress(self, enabled=False, threshold=5):
        self._progress_enabled = enabled
        if not enabled:
            self._curl.setopt(pycurl.NOPROGRESS, 1)
            return

        self._curl.setopt(pycurl.NOPROGRESS, 0)
        self._curl.setopt(pycurl.PROGRESSFUNCTION, self.progress_func)
        self._progress_update_threshold = threshold

    def progress_func(self, download_t, download_d, upload_t, upload_d):
        cur_time = time.time()
        if self._progress_update_time is None:
            self._progress_update_time = cur_time
        else:
            if cur_time - self._progress_update_time > self._progress_update_threshold:
                self._progress_update_time = cur_time
                if int(download_t) > 0:
                    logger.debug("Download progress:%s/%s bytes (%.0f%%)", download_d, download_t, 100 * download_d / download_t)
                if int(upload_t) > 0:
                    logger.debug("Upload progress:%s/%s bytes (%.0f%%)", upload_d, upload_t, 100 * upload_d / upload_t)

    def set_ip_resolve(self, resolve_type, retry_threshold=6, retry_timer=180):
        # skip if no changed
        if self._ip_resolve_type == resolve_type and \
           self._resolve_retry_threshold == retry_threshold and \
           self._resolve_retry_timer == retry_timer:
            logger.info("No change ip resolve type since current type is %s", self._ip_resolve_type)
            return

        self._resolve_retry_threshold = retry_threshold
        if resolve_type == self.IPRESOLVE_V4_FIRST or resolve_type == self.IPRESOLVE_V4_ONLY:
            self._set_resolve_v4()
        elif resolve_type == self.IPRESOLVE_V6_FIRST or resolve_type == self.IPRESOLVE_V6_ONLY:
            self._set_resolve_v6()
        elif resolve_type == self.IPRESOLVE_WHATEVER:
            self._set_resolve_whatever()
        else:
            logger.error("Unsupported ip resolving type(%s) is detected.", resolve_type)
            return
        self._ip_resolve_type = resolve_type
        self._count_resolve_change_then_success = 0
        self._resolve_retry_timer = retry_timer

    def _set_resolve_v4(self):
        self._renew_curl_object()
        self.set_option(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
        logger.info("Resolving type will be changed to IPv4 for url:%s", self.url)
        self._current_resolving_type = self.STATE_IPV4

    def _set_resolve_v6(self):
        self._renew_curl_object()
        self.set_option(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V6)
        logger.info("Resolving type will be changed to IPv6 for url:%s", self.url)
        self._current_resolving_type = self.STATE_IPV6

    def _set_resolve_whatever(self):
        self._renew_curl_object()
        self.set_option(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER)
        logger.info("Resolving type will be changed to whatever for url:%s", self.url)
        self._current_resolving_type = self.STATE_WHATEVER

    def _should_change_resolving(self, curl_err_code):
        # None for resolving type doesn't match the hostname in querying url.
        # CURLE_COULDNT_RESOLVE_HOST (6)
        # CURLE_COULDNT_CONNECT (7)
        # CURLE_OPERATION_TIMEDOUT (28)
        # COULDNT_CONNECT may occur when the network environment doesn't support IPv6,
        # even if the server does support IPv6.
        return curl_err_code in (None, 6, 7, 28) and self._in_proper_resolving()

    def _check_resolving_for_adaptive_mode(self):
        if self._resolve_change_time is None:
            return

        # cur_time = monotonictime.monotonic_time()
        cur_time = time.time()
        timediff = cur_time - self._resolve_change_time
        if timediff > self._resolve_retry_timer:
            # swtich to original resolving
            self._resolve_change_time = None
            self._count_resolve_change_then_success = 0
            self._update_resolving_to_original()
            logger.debug("After %s seconds later, change IP resolving back to %s.",
                         timediff,
                         CurlWrapper.RESOLVE_MAPPING[self._current_resolving_type])

    def _is_using_sps(self):
        # external_service_type = utility.check_string_config("Main", "AllowExtServices", "1")
        # return external_service_type in (EXTERNAL_SERVICE_LOCAL, EXTERNAL_SERVICE_LOCAL_EX)
        return False

    def _check_resolving_for_hostname(self):
        # parse_result = urlparse(self._url)
        # if self._current_resolving_type == self.STATE_IPV4:
        #     if utility.is_valid_ipv6(parse_result.hostname):
        #         return False
        # elif self._current_resolving_type == self.STATE_IPV6:
        #     if utility.is_valid_ipv4(parse_result.hostname):
        #         return False
        # elif self._current_resolving_type == self.STATE_WHATEVER:
        #     return True
        # else:
        #     logger.error("Unknown resoving type %s while checking resoving type for hostname." % self._current_resolving_type)
        #     return False
        return True

    def _check_resolving_for_proxy(self):
        if self._proxy_enabled:
            if self._proxy_type == 'socks4':
                self.set_ip_resolve(CurlWrapper.IPRESOLVE_V4_ONLY)
            else:
                self.set_ip_resolve(CurlWrapper.IPRESOLVE_WHATEVER)
        if self._is_using_sps():
            self.set_ip_resolve(CurlWrapper.IPRESOLVE_WHATEVER)

    def _update_resolving_when_finish(self):
        if self._ip_resolve_type not in (self.IPRESOLVE_V4_FIRST, self.IPRESOLVE_V6_FIRST):
            return
        # If adapptive mode is actived, stop dual stack resolving
        if self._resolve_change_time is not None:
            return

        # switch back to original resolving
        if self._count_resolve_change_then_success < self._resolve_retry_threshold:
            self._update_resolving_to_original()
        else:
            # keep current resolving until timer threhold
            logger.debug("Query DNS %s for %d times, use it as default dns query, will turn back on after %s seconds",
                         CurlWrapper.RESOLVE_MAPPING[self._current_resolving_type],
                         self._count_resolve_change_then_success,
                         self._resolve_retry_timer)
            self._resolve_change_time = time.time()

    def _update_resolving_when_fail(self):
        if self._ip_resolve_type == self.IPRESOLVE_V4_FIRST:
            if self._current_resolving_type == self.STATE_IPV4:
                self._set_resolve_v6()
        elif self._ip_resolve_type == self.IPRESOLVE_V6_FIRST:
            if self._current_resolving_type == self.STATE_IPV6:
                self._set_resolve_v4()

    def _update_resolving_to_original(self):
        if self._ip_resolve_type == self.IPRESOLVE_V4_FIRST and self._current_resolving_type != self.STATE_IPV4:
            self._set_resolve_v4()
        elif self._ip_resolve_type == self.IPRESOLVE_V6_FIRST and self._current_resolving_type != self.STATE_IPV6:
            self._set_resolve_v6()

    def _update_resolving_statistics(self):
        # If adapptive mode is actived, stop dual stack resolving
        if self._resolve_change_time is not None:
            return

        if self._in_reverse_resolving():
            # Success when change to another resolving
            self._count_resolve_change_then_success += 1
            logger.debug("Curl query success after changing to %s. Current count is %s.",
                         CurlWrapper.RESOLVE_MAPPING[self._current_resolving_type], self._count_resolve_change_then_success)
        else:
            # success in normal status, clear all count
            self._count_resolve_change_then_success = 0

    def _in_proper_resolving(self):
        return ((self._ip_resolve_type == self.IPRESOLVE_V4_FIRST and self._current_resolving_type == self.STATE_IPV4) or
                (self._ip_resolve_type == self.IPRESOLVE_V6_FIRST and self._current_resolving_type == self.STATE_IPV6))

    def _in_reverse_resolving(self):
        return ((self._ip_resolve_type == self.IPRESOLVE_V4_FIRST and self._current_resolving_type == self.STATE_IPV6) or
                (self._ip_resolve_type == self.IPRESOLVE_V6_FIRST and self._current_resolving_type == self.STATE_IPV4))

    def _set_options(self):
        # set common options
        self._curl.setopt(pycurl.FOLLOWLOCATION, 1)
        self._curl.setopt(pycurl.NOSIGNAL, 1)
        # self._curl.setopt(pycurl.VERBOSE, 1)
        # if (self._use_default_ssl_version or
        #         utility.check_numeric_config('Main', 'SSLVersion', 0, 0, 1) == 0 or
        #         self._curl_version_major <= 6 or (self._curl_version_major == 7 and self._curl_version_minor <= 34)):
            # set pycurl.SSLVERSION to pycurl.SSLVERSION_TLSv1 when configuration SSLVersion is 0 or libcurl version <= 7.34.0
        #     self._curl.setopt(pycurl.SSLVERSION, pycurl.SSLVERSION_TLSv1)
        # else:
        #     self._curl.setopt(pycurl.SSLVERSION, pycurl.SSLVERSION_TLSv1_2)
        self._curl.setopt(pycurl.DEBUGFUNCTION, self.debug_func)

        # set specific options
        self._curl.setopt(pycurl.TIMEOUT, self._total_timeout)
        self._curl.setopt(pycurl.CONNECTTIMEOUT, self._connect_timeout)

        for k, v in self._options.iteritems():
            self._curl.setopt(k, v)

    def _set_header(self):
        # set header with NTLM workaround
        if self._header_list is None:
            hlist = []
        else:
            hlist = list(self._header_list)

        # Apply workaround only when cURL version is 7.28.0;
        # Otherwise, incorrect cURL error message "Unknown SSL protocol error in connection to grid-global.trendmicro.com:443" is retrun by libcURL
        # when the proxy type is NTLM proxy
        if self._is_ntlm and self._curl_version_major == 7 and self._curl_version_minor == 28 and self._curl_version_fix == 0:
            logger.debug("Using NTLM proxy, set http header \"Connection: Close\" to workaround curl's bug.")
            hlist.append("Connection: Close")
        else:
            hlist.append("Connection: Keep-Alive")
        self._curl.setopt(pycurl.HTTPHEADER, hlist)

    def _renew_curl_object(self):
        if isinstance(self._url, unicode):
            self._url = self._url.encode('utf-8')

        self._curl.close()
        self._curl = pycurl.Curl()
        self._change_proxy = True
        self._set_proxy_setting()
        self._set_options()
        if self._url is not None:
            self._curl.setopt(pycurl.URL, self._url)
        logger.debug("Curl object has been renew for %s." % self._url)

    @staticmethod
    def _get_protocol_from_url(url):
        from usandbox.common.urlparseipv6 import urlparse
        url_obj = urlparse(url)
        proto = url_obj.scheme
        return proto if proto else 'http'

    def should_retry_by_error_code(self, err_code, err_msg):
        if err_code in (1, 3, 4, 5, 9, 21, 22, 23, 30, 36, 37,
                        41, 42, 43, 47, 48, 49, 51, 53, 54, 58, 59, 60,
                        61, 62, 63, 67, 68, 69, 71, 73, 76, 77, 78,
                        82, 83, 84):
            return False

        if err_code == 6:
            if err_msg.find("Resolving host timed out") >= 0:
                # (6) Resolving host timed out: www.trendmicro.tw
                return True
            else:
                # (6) Could not resolve host: www.trendmicro.taipei (Domain name not found)
                # (6) Could not resolve host: www.trendmicro.tw (Could not contact DNS servers)
                return False

        if err_code == 7:
            if err_msg.find("User was rejected by the SOCKS5 server") >= 0 or\
               err_msg.find("Connection refused") >= 0 or\
               err_msg.find("No route to host") >= 0 or\
               err_msg.find("couldn't connect to host") >= 0 or\
               err_msg.find("Address family not supported by protocol") >= 0:
                return False
            else:
                return True

        if err_code == 28:
            if err_msg.find("Resolving timed out") >= 0 or\
               err_msg.find("Operation timed out after 0 milliseconds with 0 out of 0 bytes received") >= 0:
                # (28) Resolving timed out after 10000 milliseconds
                # (28) Operation timed out after 0 milliseconds with 0 out of 0 bytes received
                return True
            else:
                return False

        if err_code == 56:
            if err_msg.find("Received HTTP code 407 from proxy after CONNECT") >= 0:
                return False
            else:
                return True

        return True

    def _workaround_ntlm_proxy(self):
        def __write_header_normal(s):
            # This is not NTLM related feature, but default header function
            # is implemented with NTLM workaround, so put this here.
            if "X-SPS-Result" in s:
                logger.debug("%s", s.strip())

            if pycurl.HEADERFUNCTION in self._options:
                return self._options[pycurl.HEADERFUNCTION](s)
            else:
                return len(s)

        def __write_header(s):
            if self.RE_PATTERN_IS_NTLM_PROXY.search(s):
                self._pycurl_23_caused_by_ntlm = True
                return -1
            return __write_header_normal(s)

        if self._is_ntlm:
            # workaround for libcurl will enter infinite loop while using NTLM proxy in libcurl 7.39.0
            # we need to reset pycurl object to workaround this issue...
            logger.debug("Renew curl object to workaround curl infinite loop bug when using NTLM.")
            self._renew_curl_object()
            self._curl.setopt(pycurl.HEADERFUNCTION, __write_header_normal)
        else:
            self._curl.setopt(pycurl.HEADERFUNCTION, __write_header)

    def close(self):
        # This function is mainly for breaking the cyclic references between CurlWrapper and self._curl object
        self._curl.close()
        self._curl = None
        self._options.clear()

    def set_add_to_job_queue_time(self):
        self._add_to_job_queue_time = time.time()

    def get_add_to_job_queue_time(self):
        return self._add_to_job_queue_time

    def flush_cookie_to_file(self):
        self.set_option(pycurl.COOKIELIST, "FLUSH")


# class MultithreadingCurlWrapper(object):
#     def __init__(self, *args, **kwargs):
#         self._curl_pool = {}
#         self._init_kwargs = kwargs
#         self._init_args = args

#         # Globbal proxy setting
#         self._proxy_enabled = False
#         self._proxy_host = None
#         self._proxy_port = None
#         self._proxy_user = None
#         self._proxy_passwd = None
#         self._proxy_type = None

#         tid = thread.get_ident()
#         if tid not in self._curl_pool:
#             self._curl_pool[tid] = CurlWrapper(*args, **kwargs)

#     def __getattr__(self, name):
#         tid = thread.get_ident()
#         if tid not in self._curl_pool:
#             curl_wrapper = CurlWrapper(*self._init_args, **self._init_kwargs)
#             curl_wrapper.set_proxy(enabled=self._proxy_enabled,
#                                    proxy_host=self._proxy_host,
#                                    proxy_port=self._proxy_port,
#                                    proxy_type=self._proxy_type,
#                                    proxy_user=self._proxy_user,
#                                    proxy_passwd=self._proxy_passwd)
#             self._curl_pool[tid] = curl_wrapper

#         return getattr(self._curl_pool[tid], name)

#     @property
#     def proxy_enabled(self):
#         return self._proxy_enabled

#     def close(self, thread_id=None):
#         tid = thread_id or thread.get_ident()
#         if tid in self._curl_pool:
#             del self._curl_pool[tid]

#     def close_all(self):
#         del self._curl_pool
#         self._curl_pool = {}

#     def get_length(self):
#         return len(self._curl_pool)

#     def set_proxy_to_all(self, enabled, proxy_host=None, proxy_port=None, proxy_type=None, proxy_user=None, proxy_passwd=None):
#         self._proxy_enabled = enabled
#         self._proxy_host = proxy_host
#         self._proxy_port = proxy_port
#         self._proxy_type = proxy_type
#         self._proxy_user = proxy_user
#         self._proxy_passwd = proxy_passwd

#         for _, curl_wrapper in self._curl_pool.iteritems():
#             curl_wrapper.set_proxy(enabled=enabled,
#                                    proxy_host=proxy_host,
#                                    proxy_port=proxy_port,
#                                    proxy_type=proxy_type,
#                                    proxy_user=proxy_user,
#                                    proxy_passwd=proxy_passwd)


class CurlWrapperDispathcer(object):

    def __init__(self, curlwrapper_job_queue):
        self._curlmulti = pycurl.CurlMulti()
        self._stopping = False
        self._current_job_dict = {}
        self._curlwrapper_job_queue = curlwrapper_job_queue
        self._max_job_in_curlmulti = 3
        self._select_timeout = 0.1

    def reload_config(self):
        self._max_job_in_curlmulti = 1
        self._select_timeout = 0.1
        logger.debug("update curlwrapper dispatcher MaxJobInCurlMulti:%s, MaxTimeInCurlMultiSelect:%s" % (self._max_job_in_curlmulti, self._select_timeout))

    def _add_job(self, curlwrapper_obj):
        curlwrapper_obj.pre_preform(curlwrapper_obj.url, curlwrapper_obj.header_list)
        self._curlwrapper_job_queue.put(curlwrapper_obj)

    def _setup(self):
        # logger.debug('get curlwrapper from queue {}'.format(self._current_job_dict))
        current_job_num = len(self._current_job_dict)

        if current_job_num >= self._max_job_in_curlmulti:
            return

        for _ in xrange(current_job_num, self._max_job_in_curlmulti):
            try:
                # logger.debug('get curlwrapper from queue')
                curlwrapper_object = self._curlwrapper_job_queue.get_nowait()
                logger.debug("Found a curlwrapper job:%s, current running job number:%s, current waiting job number:%s, priority:%s",
                             curlwrapper_object.url,
                             current_job_num,
                             self._curlwrapper_job_queue.qsize(),
                             curlwrapper_object.priority)
                curlwrapper_object.setup_curl()

                _curl = curlwrapper_object.get_curl_object()

                self._curlmulti.add_handle(_curl)
                self._current_job_dict[_curl] = curlwrapper_object

            except Queue.Empty:
                break

    def _teardown(self, curl_obj, errno=None, errmsg=None):
        curlwrapper_obj = self._current_job_dict.pop(curl_obj, None)
        _effective_url = curl_obj.getinfo(pycurl.EFFECTIVE_URL)
        _url = curlwrapper_obj.url
        _uuid = curlwrapper_obj.uuid

        def _analyze_result_for_retry():
            if errno:
                logger.debug("url(%s) encountered errno(%s), errmsg(%s)" % (curl_obj.getinfo(pycurl.EFFECTIVE_URL), errno, errmsg))
                retry = curlwrapper_obj.pycurl_error_handling(errno, errmsg)
                return retry
            else:
                curlwrapper_obj.curl_response = CurlResponse(http_code=curlwrapper_obj.http_code, server_ip=curlwrapper_obj.primary_ip)

                if curlwrapper_obj.need_retry:
                    logger.debug("Receive http code %s, retry (%s/%s), it should retry.", curlwrapper_obj.http_code, curlwrapper_obj._reties, curlwrapper_obj._max_retry_times)
                    curlwrapper_obj._reties += 1
                    return True
                logger.debug("[CurlMulti] finished: %s, is_succeed:%s, http_code:%s, url(%s), effective_url(%s)" % (curlwrapper_obj.url, curlwrapper_obj.is_succeed, curlwrapper_obj.http_code, curlwrapper_obj.curl_response.url, curlwrapper_obj.curl_response.effective_url))
                return False

        # put the job back into curlmulti for retry
        if not (not _analyze_result_for_retry() or curlwrapper_obj.is_done or curlwrapper_obj.is_succeed):
            logger.debug("[CurlMulti] retry perform for url:%s, http_code(%s), error_cod(%s), error_msg(%s)", curlwrapper_obj.url, curlwrapper_obj.http_code, curlwrapper_obj.curl_err_code, curlwrapper_obj.curl_err_msg)
            self._add_job(curlwrapper_obj)
            return

        # provide the information for Redirector used
        curlwrapper_obj.curl_response.effective_url = _effective_url
        curlwrapper_obj.curl_response.url = _url
        curlwrapper_obj.curl_response.uuid = _uuid

        # call back
        logger.debug("[CurlMulti] curl job is done with response(%s)" % curlwrapper_obj.curl_response.__dict__)
        print(curlwrapper_obj)
        curlwrapper_obj.callback_done(curlwrapper_obj)

    def perform(self):
        try:
            self._setup()

            if not self._current_job_dict:
                time.sleep(0.1)

            # Perform multi-request.
            # This code copied from pycurl docs, modified to explicitly
            # set num_handles before the outer while loop.
            # It is an inefficient way, make too much function call
            # TODO: design an efficient structure if possible
            if self._curlmulti.select(self._select_timeout) == -1:
                return

            # Since something's waiting, give curl a chance to process it
            while True:
                # during performing cannot add handler
                # TODO: add lock
                ret, num_handles = self._curlmulti.perform()
                if ret != pycurl.E_CALL_MULTI_PERFORM:
                    break

            while True:
                num_q, ok_list, err_list = self._curlmulti.info_read()
                for curl_obj in ok_list:
                    self._curlmulti.remove_handle(curl_obj)
                    self._teardown(curl_obj)

                for curl_obj, errno, errmsg in err_list:
                    self._curlmulti.remove_handle(curl_obj)
                    self._teardown(curl_obj, errno, errmsg)

                if num_q < 1:
                    break

        except BaseException:
            import traceback
            logger.error("Unknown exception occured in CurlWrapperDispathcer CurlMulti {}".format(traceback.format_exc()))
            sys.exit(1)


class PrefetchCurlWrapperDispathcer(threading.Thread):
    def __init__(self, thread_name, curlwrapper_job_queue):
        threading.Thread.__init__(self, name=thread_name)
        self._pqm = curlwrapper_job_queue
        self._stopping = False
        self._curlwrapper_dispatcher = CurlWrapperDispathcer(self._pqm)

    def run(self):
        logger.debug("%s starts to run.", self.name)
        while not self._stopping:
            try:
                self._curlwrapper_dispatcher.perform()
            except BaseException:
                logger.error("Unknown exception occured in PrefetchCurlWrapperDispathcer")

        logger.debug("%s ends.", self.name)

    def stop(self):
        self._stopping = True


class CurlTask(object):

    actionType = None
    payload = None
    callback_done = None

if  __name__ == "__main__":
    # import sys
    logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format='%(asctime)s %(levelname)-8s[%(filename)s:%(lineno)d(%(funcName)s)] %(message)s')

    import Queue
    curl_job_queue = Queue.Queue()
    download_thread_works = []
    # Create working threads
    for i in xrange(1):
        logger.info("Prefetchd download dispatcher thread number: %d" %i)
        download_thread = PrefetchCurlWrapperDispathcer("Prefetcher CurlWrapper Dispathcer_%d" % i, curl_job_queue)
        download_thread.start()
        download_thread_works.append(download_thread)

    for url in ['http://baidu.com', 'http://google.com']:
        curl_wrapper = CurlWrapper(600, 30)
        curl_wrapper.url = url
        curl_wrapper._curl.setopt(pycurl.URL, curl_wrapper.url)
        curl_wrapper.callback_done = lambda a: logger.debug('checking download result {}'.format(a))
        curl_job_queue.put(curl_wrapper)

